﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace BuildSDFDatabase
{
	class ResearchDataParser
	{
		private int fieldID;
		private string corporaMetaPath;
		private string folderName;
		private string baseCorpus;
		public DateTime startTime, endTime;

		public Dictionary<string, int> Corpora;
		public Dictionary<string, int> Fields;
		public Dictionary<string, int> Venues;
		public Dictionary<string, int> Organizations;
		public Dictionary<string, int> Programs;
		public Dictionary<string, int> Authors;

		public DataTable CorporaDT { get; private set; }
		public DataTable FieldDT { get; private set; }

		public DataTable ClusterInfoDT { get; private set; }
		public DataTable CorpusTimeClusterDT { get; private set; }
		public DataTable CorpusTimeClusterKeywordDT { get; private set; }

		public DataTable ProposalDocumentDT { get; private set; }
		public DataTable PaperDocumentDT { get; private set; }
		public DataTable VenueDT { get; private set; }
		public DataTable OrganizationDT { get; private set; }
		public DataTable ProgramDT { get; private set; }
		public DataTable AuthorDT { get; private set; }
		public DataTable AuthorShipDT { get; private set; }
		public DataTable ClusterDocumentDT { get; private set; }

		public DataTable RoseTreeDT { get; private set; }
		public DataTable GlobalLeadLagDT { get; private set; }
		public DataTable LocalLeadLagDT { get; private set; }
		public DataTable DocumentMatrixDT { get; private set; }

		public ResearchDataParser(string corporaMetaPath)
		{
			this.corporaMetaPath = corporaMetaPath;
			LoadCorporaMeta();
		}

		public List<DataTable> GetCoporaTables()
		{
			List<DataTable> tables = new List<DataTable>();
			tables.Add(CorporaDT);
			tables.Add(FieldDT);
			return tables;
		}
		public List<DataTable> GetClusterTables()
		{
			List<DataTable> tables = new List<DataTable>();
			tables.Add(ClusterInfoDT);
			tables.Add(CorpusTimeClusterDT);
			tables.Add(CorpusTimeClusterKeywordDT);
			return tables;
		}
		public List<DataTable> GetDocumentTables()
		{
			List<DataTable> tables = new List<DataTable>();
			tables.Add(ProposalDocumentDT);
			tables.Add(PaperDocumentDT);
			tables.Add(VenueDT);
			tables.Add(OrganizationDT);
			tables.Add(ProgramDT);
			tables.Add(AuthorDT);
			tables.Add(AuthorShipDT);
			tables.Add(ClusterDocumentDT);
			return tables;
		}
		public List<DataTable> GetLeadLagTables()
		{
			List<DataTable> tables = new List<DataTable>();
			tables.Add(RoseTreeDT);
			tables.Add(GlobalLeadLagDT);
			tables.Add(LocalLeadLagDT);
			tables.Add(DocumentMatrixDT);
			return tables;
		}

		#region CorporaMeta Loader

		private void LoadCorporaMeta()
		{
			SetupCorporaMetaDataTables();
			SetupClusterMetaDataTables();
			SetupDocumentTables();
			SetupLeadLagTables();

			Corpora = new Dictionary<string, int>();
			Fields = new Dictionary<string, int>();
			Venues = new Dictionary<string, int>();
			Organizations = new Dictionary<string, int>();
			Programs = new Dictionary<string, int>();
			Authors = new Dictionary<string, int>();

			using (StreamReader reader = new StreamReader(Path.Combine(corporaMetaPath, "corpora_meta.txt")))
			{
				var line = reader.ReadLine();
				var times = line.Split(' ').ToArray();
				startTime = IntToDateTime(int.Parse(times[0]));
				endTime = IntToDateTime(int.Parse(times[1]));

				reader.ReadLine();
				int corpusIndex = 0;
				while (true)
				{
					var corpus = reader.ReadLine().Trim();
					if (string.IsNullOrEmpty(corpus))
						break;
					CorporaDT.Rows.Add(corpusIndex, corpus);
					Corpora.Add(corpus, corpusIndex);
					if (corpusIndex == 0)
						baseCorpus = corpus;
					corpusIndex++;
				}
				int fieldIndex = 0;
				while (true)
				{
					var field = reader.ReadLine();
					if (string.IsNullOrEmpty(field))
						break;
					FieldDT.Rows.Add(fieldIndex, field.Trim());
					Fields.Add(field.Trim(), fieldIndex);
					fieldIndex++;
				}
			}
		}

		private void SetupCorporaMetaDataTables()
		{
			CorporaDT = new DataTable("Corpora");
			CorporaDT.Columns.Add(new DataColumn("CorpusID", typeof(int)));
			CorporaDT.Columns.Add(new DataColumn("CorpusName", typeof(string)));

			FieldDT = new DataTable("Field");
			FieldDT.Columns.Add(new DataColumn("FieldID", typeof(int)));
			FieldDT.Columns.Add(new DataColumn("FieldName", typeof(string)));
		}
		#endregion

		#region ClusterMeta Loader
		public void LoadClusterMeta(string field)
		{
			this.folderName = Path.Combine(corporaMetaPath, field);
			this.fieldID = Fields[field];
			ClusterTableClear();

			var dataFolderName = Path.Combine(folderName, "GlobalSplitMerge");
			if (!Directory.Exists(dataFolderName))
				throw new Exception("ParseGlobalSplitMerge: data folder does not exist.");

			string[] filenames = Directory.GetFiles(dataFolderName);
			foreach (var filename in filenames)
			{
				if(filename.EndsWith("cluster.txt"))
					ParseGlobalSplitMergeCluster(filename);
			}
		}

		private void SetupClusterMetaDataTables()
		{
			//clusterInfoDT: Field, ClusterID, Timestamp, Size
			ClusterInfoDT = new DataTable("ClusterInfo");
			DataColumn[] primaryKey0 = new DataColumn[5];
			primaryKey0[0] = new DataColumn("FieldID", typeof(int));
			primaryKey0[1] = new DataColumn("ClusterID", typeof(int));
			primaryKey0[2] = new DataColumn("Timestamp", typeof(DateTime));
			ClusterInfoDT.Columns.Add(primaryKey0[0]);
			ClusterInfoDT.Columns.Add(primaryKey0[1]);
			ClusterInfoDT.Columns.Add(primaryKey0[2]);
			ClusterInfoDT.Columns.Add(new DataColumn("Size", typeof(int)));

			//clusterCorpusDT: Field ClusterID, Timestamp, Corpus, Size
			CorpusTimeClusterDT = new DataTable("CorpusTimeCluster");
			CorpusTimeClusterDT.Columns.Add(new DataColumn("FieldID", typeof(int)));
			CorpusTimeClusterDT.Columns.Add(new DataColumn("ClusterID", typeof(int)));
			CorpusTimeClusterDT.Columns.Add(new DataColumn("Timestamp", typeof(DateTime)));
			CorpusTimeClusterDT.Columns.Add(new DataColumn("CorpusID", typeof(int)));
			CorpusTimeClusterDT.Columns.Add(new DataColumn("Size", typeof(int)));

			//clusterCorpusKeywordDT: Field, ClusterID, Timestamp, Corpus, Keyword, Frequency
			CorpusTimeClusterKeywordDT = new DataTable("CorpusTimeClusterKeyword");
			DataColumn[] primaryKey = new DataColumn[5];
			primaryKey[0] = new DataColumn("FieldID", typeof(int));
			primaryKey[1] = new DataColumn("ClusterID", typeof(int));
			primaryKey[2] = new DataColumn("Timestamp", typeof(DateTime));
			primaryKey[3] = new DataColumn("CorpusID", typeof(int));
			primaryKey[4] = new DataColumn("Keyword", typeof(string));
			CorpusTimeClusterKeywordDT.Columns.Add(primaryKey[0]);
			CorpusTimeClusterKeywordDT.Columns.Add(primaryKey[1]);
			CorpusTimeClusterKeywordDT.Columns.Add(primaryKey[2]);
			CorpusTimeClusterKeywordDT.Columns.Add(primaryKey[3]);
			CorpusTimeClusterKeywordDT.Columns.Add(primaryKey[4]);
			CorpusTimeClusterKeywordDT.Columns.Add(new DataColumn("Frequency", typeof(int)));
			CorpusTimeClusterKeywordDT.PrimaryKey = primaryKey;
		}

		private void ClusterTableClear()
		{
			ClusterInfoDT.Rows.Clear();
			CorpusTimeClusterDT.Rows.Clear();
			CorpusTimeClusterKeywordDT.Rows.Clear();
		}

		private void ParseGlobalSplitMergeCluster(string filename)
		{
			using (StreamReader reader = new StreamReader(filename))
			{
				int time = int.Parse(reader.ReadLine().Trim());
				var timestamp = IntToDateTime(time);
				reader.ReadLine();
				reader.ReadLine();
				while (true)
				{
					var line = reader.ReadLine();
					if (string.IsNullOrEmpty(line) || string.IsNullOrWhiteSpace(line))
						break;
					var contents = line.Split('|').Select(t => t.Trim()).ToArray();
					if (contents.Length < 2)
						throw new Exception("ParseGlobalSplitMergeCluster: file format error.");
					int clusterId = int.Parse(contents[0]);
					int size = int.Parse(contents[1]);
					//cluster info add
					ClusterInfoDT.Rows.Add(fieldID, clusterId, timestamp, size);
				}

				reader.ReadLine();
				while (true)
				{
					var line = reader.ReadLine();
					if (string.IsNullOrEmpty(line) || string.IsNullOrWhiteSpace(line))
						break;
					var contents = line.Split('|').Select(t => t.Trim()).ToArray();
					if (contents.Length != 3)
						throw new Exception("ParseGlobalSplitMergeCluster: file format error.");
					int clusterId = int.Parse(contents[0]);
					foreach (var corpusInfo in contents[2].Split('\t'))
					{
						if (corpusInfo.Length > 0)
						{
							var corpusArray = corpusInfo.Split(':');
							string corpus = corpusArray[0];
							int corpusID = Corpora[corpus];
							int size = int.Parse(corpusArray[1]);
							//cluster corpus add
							CorpusTimeClusterDT.Rows.Add(fieldID, clusterId, timestamp, corpusID, size);
						}
					}
				}

				while (true)
				{
					var line = reader.ReadLine();
					if (line == null)
						break;
					var contents = line.Split(':');
					int clusterId = int.Parse(contents[1]);
					while (true)
					{
						line = reader.ReadLine();
						if (string.IsNullOrEmpty(line) || string.IsNullOrWhiteSpace(line))
							break;
						int firstColonIndex = line.IndexOf(':');
						string corpus = line.Substring(0, firstColonIndex);
						int corpusID = Corpora[corpus];
						string keywordMeta = line.Substring(firstColonIndex + 1);
						string[] metaData = keywordMeta.Split(';').Select(t => t.Trim()).ToArray();
						foreach (var meta in metaData)
						{
							if (meta.Length > 0)
							{
								string[] wordContent = meta.Split(':');
								string keyword = wordContent[0];
								int frq = int.Parse(wordContent[1]);
								//cluster corpus keyword add
								CorpusTimeClusterKeywordDT.Rows.Add(fieldID, clusterId, timestamp, corpusID, keyword, frq);
							}
						}
					}
				}
			}
		}
		#endregion

		#region Document Loader
		public void LoadDocument(string field)
		{
			this.folderName = Path.Combine(corporaMetaPath, field);
			this.fieldID = Fields[field];
			DocumentTablesClear();

			var docFolders = Directory.GetDirectories(folderName);

			foreach (var folder in docFolders)
			{
				if (folder.EndsWith("Doc"))
				{
					ArchiveCorpusDocument(folder);
				}
			}
		}

		private void ArchiveCorpusDocument(string folder)
		{
			var array = folder.Split('\\').ToArray();
			string corpus = array[array.Length - 1].Replace("Doc", "");

			var files = Directory.GetFiles(folder);
			foreach (var file in files)
			{
				ArchiveClusterCorpusDocument(file, corpus);
			}
		}

		private void ArchiveClusterCorpusDocument(string file, string corpus)
		{
			int corpusID = Corpora[corpus];
			var array = file.Split('\\').ToArray();
			int clusterId = int.Parse(array[array.Length - 1].Replace("cluster_", "").Replace(".txt", ""));

			using (StreamReader reader = new StreamReader(file))
			{
				reader.ReadLine();
				reader.ReadLine();
				while (true)
				{
					var line = reader.ReadLine();
					if (string.IsNullOrEmpty(line))
						break;
					var contents = line.Split('|');
					if (contents.Length < 5)
						throw new Exception("ArchiveClusterCorpusDocument: file format error.");

					int time = int.Parse(contents[0]);
					var timestamp = IntToDateTime(time);
					int documentId = int.Parse(contents[1]);
					string title = contents[2].Trim();
					string abstr = contents[3].Trim();
					string readCorpus = contents[4];
					if(corpus != readCorpus)
						throw new Exception("ArchiveClusterCorpusDocument: corpus does not match.");

					//db add
					ClusterDocumentDT.Rows.Add(documentId, fieldID, clusterId, corpusID);
					//properties add
					if (corpus == CorpusName.Paper)
					{
						string keywords = contents[5].Trim();
						string venue = contents[6].Trim().ToUpper();
						string authors = contents[7].Trim();
						int venueID = -1;
						if (!Venues.TryGetValue(venue, out venueID))
						{
							venueID = Venues.Count;
							Venues.Add(venue, venueID);
							VenueDT.Rows.Add(venueID, venue);
						}
						PaperDocumentDT.Rows.Add(documentId, timestamp, title, abstr, corpusID, keywords, venueID);

						var field = "";
						foreach(var pair in Fields)
						{
							if(pair.Value == fieldID)
							{
								field = pair.Key;
								break;
							}
						}
						var authorList = AuthorNameParser.GetPaperAuthors(field, authors);
						if (authorList != null && authorList.Count > 0)
						{
							foreach (var author in authorList.Distinct())
								InsertAuthorTable(documentId, author, authorList.IndexOf(author));
						}
					}
					else if (corpus == CorpusName.Proposal)
					{
						string organization = contents[6].Trim();
						string pi = contents[7].Trim();
						string copis = contents[8].Trim();
						int awardedAmount = int.Parse(contents[9].Trim());
						string program = contents[10].Trim().ToUpper();

						int orgID = -1;
						if (!Organizations.TryGetValue(organization, out orgID))
						{
							orgID = Organizations.Count;
							Organizations.Add(organization, orgID);
							OrganizationDT.Rows.Add(orgID, organization);
						}
						int prgID = -1;
						if (!Programs.TryGetValue(program, out prgID))
						{
							prgID = Programs.Count;
							Programs.Add(program, prgID);
							ProgramDT.Rows.Add(prgID, program);
						}
						ProposalDocumentDT.Rows.Add(documentId, timestamp, title, abstr, corpusID, orgID, awardedAmount, prgID);

						var piName = AuthorNameParser.GetPI(pi);
						InsertAuthorTable(documentId, piName, 0);

						var copiNames = AuthorNameParser.GetCoPI(copis);
						if(copiNames != null)
						{
							foreach (var copiName in copiNames)
								InsertAuthorTable(documentId, copiName, 1 + copiNames.IndexOf(copiName));
						}
					}
				}
			}
		}

		private void InsertAuthorTable(int documentId, Tuple<string, string> author, int authorRank)
		{
			var firstName = author.Item1;
			var lastName = author.Item2;
			var name = firstName.ToLower() + " " + lastName.ToLower();
			int authorID = -1;
			if (!Authors.TryGetValue(name, out authorID))
			{
				authorID = Authors.Count;
				Authors.Add(name, authorID);
				AuthorDT.Rows.Add(authorID, firstName, lastName);
			}
			AuthorShipDT.Rows.Add(documentId, authorID, authorRank);
		}

		//private List<Tuple<string, string>> GetPaperAuthors(string field, string authors);
		//private Tuple<string, string> GetPI(string pi);
		//private List<Tuple<string, string>> GetCoPI(string copi);

		private void SetupDocumentTables()
		{
			PaperDocumentDT = new DataTable("Document");
			PaperDocumentDT.Columns.Add(new DataColumn("DocumentID", typeof(int)));
			PaperDocumentDT.Columns.Add(new DataColumn("Timestamp", typeof(DateTime)));
			PaperDocumentDT.Columns.Add(new DataColumn("Title", typeof(string)));
			PaperDocumentDT.Columns.Add(new DataColumn("Abstract", typeof(string)));
			PaperDocumentDT.Columns.Add(new DataColumn("CorpusID", typeof(int)));
			PaperDocumentDT.Columns.Add(new DataColumn("Keywords", typeof(string)));
			PaperDocumentDT.Columns.Add(new DataColumn("VenueID", typeof(int)));

			ProposalDocumentDT = new DataTable("Document");
			ProposalDocumentDT.Columns.Add(new DataColumn("DocumentID", typeof(int)));
			ProposalDocumentDT.Columns.Add(new DataColumn("Timestamp", typeof(DateTime)));
			ProposalDocumentDT.Columns.Add(new DataColumn("Title", typeof(string)));
			ProposalDocumentDT.Columns.Add(new DataColumn("Abstract", typeof(string)));
			ProposalDocumentDT.Columns.Add(new DataColumn("CorpusID", typeof(int)));
			ProposalDocumentDT.Columns.Add(new DataColumn("NSFOrganizationID", typeof(int)));
			ProposalDocumentDT.Columns.Add(new DataColumn("AwardedAmountToDate", typeof(int)));
			ProposalDocumentDT.Columns.Add(new DataColumn("NSFProgramID", typeof(int)));

			VenueDT = new DataTable("Venue");
			VenueDT.Columns.Add(new DataColumn("VenueID", typeof(int)));
			VenueDT.Columns.Add(new DataColumn("VenueName", typeof(string)));

			OrganizationDT = new DataTable("NSFOrganization");
			OrganizationDT.Columns.Add(new DataColumn("NSFOrganizationID", typeof(int)));
			OrganizationDT.Columns.Add(new DataColumn("NSFOrganizationName", typeof(string)));

			ProgramDT = new DataTable("NSFProgram");
			ProgramDT.Columns.Add(new DataColumn("NSFProgramID", typeof(int)));
			ProgramDT.Columns.Add(new DataColumn("NSFProgramName", typeof(string)));

			AuthorDT = new DataTable("Author");
			AuthorDT.Columns.Add(new DataColumn("AuthorID", typeof(int)));
			AuthorDT.Columns.Add(new DataColumn("FirstName", typeof(string)));
			AuthorDT.Columns.Add(new DataColumn("LastName", typeof(string)));

			AuthorShipDT = new DataTable("AuthorShip");
			DataColumn[] primaryKey = new DataColumn[2];
			primaryKey[0] = new DataColumn("DocumentID", typeof(int));
			primaryKey[1] = new DataColumn("AuthorID", typeof(int));
			AuthorShipDT.Columns.Add(primaryKey[0]);
			AuthorShipDT.Columns.Add(primaryKey[1]);
			AuthorShipDT.Columns.Add(new DataColumn("Rank", typeof(int)));
			AuthorShipDT.PrimaryKey = primaryKey;

			ClusterDocumentDT = new DataTable("ClusterDocument");
			ClusterDocumentDT.Columns.Add(new DataColumn("DocumentID", typeof(int)));
			ClusterDocumentDT.Columns.Add(new DataColumn("FieldID", typeof(int)));
			ClusterDocumentDT.Columns.Add(new DataColumn("ClusterID", typeof(int)));
			ClusterDocumentDT.Columns.Add(new DataColumn("CorpusID", typeof(int)));
		}

		private void DocumentTablesClear()
		{
			PaperDocumentDT.Rows.Clear();
			ProposalDocumentDT.Rows.Clear();
			VenueDT.Rows.Clear();
			OrganizationDT.Rows.Clear();
			ProgramDT.Rows.Clear();
			AuthorDT.Rows.Clear();
			AuthorShipDT.Rows.Clear();
			ClusterDocumentDT.Rows.Clear();
		}
		#endregion

		#region LeadLag Loader

		public void LoadLeadLagMeta(string field)
		{
			this.folderName = Path.Combine(corporaMetaPath, field);
			this.fieldID = Fields[field];
			LeadLagTablesClear();

			var roseTreeFile = Path.Combine(folderName, "outputTree.txt");
			LoadRoseTreeMeta(roseTreeFile);

			var files = Directory.GetFiles(folderName);
			foreach (var file in files)
			{
				var array = file.Split('\\');
				var filename = array[array.Length - 1];
				if (filename.StartsWith("GlobalLeadLagSummary"))
				{
					var names = filename.Replace(".txt", "").Split('_');
					string corpus1 = names[1];
					string corpus2 = names[2];
					LoadGlobalLeadLagMeta(corpus1, corpus2);
					LoadLocalLeadLagMeta(corpus1, corpus2);
					LoadDocumentMatrix(corpus1, corpus2);
				}
			}
		}

		private void SetupLeadLagTables()
		{
			RoseTreeDT = new DataTable("RoseTree");
			RoseTreeDT.Columns.Add(new DataColumn("FieldID", typeof(int)));
			RoseTreeDT.Columns.Add(new DataColumn("Child", typeof(string)));
			RoseTreeDT.Columns.Add(new DataColumn("Parent", typeof(string)));

			GlobalLeadLagDT = new DataTable("GlobalLeadLag");
			GlobalLeadLagDT.Columns.Add(new DataColumn("FieldID", typeof(int)));
			GlobalLeadLagDT.Columns.Add(new DataColumn("ObjectID", typeof(string)));
			GlobalLeadLagDT.Columns.Add(new DataColumn("CorpusID", typeof(int)));
			GlobalLeadLagDT.Columns.Add(new DataColumn("LeadLagValue", typeof(double)));

			LocalLeadLagDT = new DataTable("LocalLeadLag");
			LocalLeadLagDT.Columns.Add(new DataColumn("FieldID", typeof(int)));
			LocalLeadLagDT.Columns.Add(new DataColumn("ClusterID", typeof(int)));
			LocalLeadLagDT.Columns.Add(new DataColumn("Timestamp", typeof(DateTime)));
			LocalLeadLagDT.Columns.Add(new DataColumn("CorpusID1", typeof(int)));
			LocalLeadLagDT.Columns.Add(new DataColumn("CorpusID2", typeof(int)));
			LocalLeadLagDT.Columns.Add(new DataColumn("Value1", typeof(double)));
			LocalLeadLagDT.Columns.Add(new DataColumn("Value2", typeof(double)));

			DocumentMatrixDT = new DataTable("DocumentMatrix");
			DocumentMatrixDT.Columns.Add(new DataColumn("FieldID", typeof(int)));
			DocumentMatrixDT.Columns.Add(new DataColumn("ClusterID", typeof(int)));
			DocumentMatrixDT.Columns.Add(new DataColumn("CorpusID1", typeof(int)));
			DocumentMatrixDT.Columns.Add(new DataColumn("CorpusID2", typeof(int)));
			DocumentMatrixDT.Columns.Add(new DataColumn("DocumentID1", typeof(int)));
			DocumentMatrixDT.Columns.Add(new DataColumn("DocumentID2", typeof(int)));
			DocumentMatrixDT.Columns.Add(new DataColumn("Similarity", typeof(double)));
		}

		private void LeadLagTablesClear()
		{
			RoseTreeDT.Rows.Clear();
			GlobalLeadLagDT.Rows.Clear();
			LocalLeadLagDT.Rows.Clear();
			DocumentMatrixDT.Rows.Clear();
		}
		private void LoadRoseTreeMeta(string roseTreeFile)
		{
			using (StreamReader reader = new StreamReader(roseTreeFile))
			{
				while (true)
				{
					var line = reader.ReadLine();
					if (string.IsNullOrEmpty(line))
						break;
					var objects = line.Split('-', '>').Select(t => t.Trim()).Where(t => t.Length > 0).ToArray();
					for (int i = 0; i < objects.Length - 1; i++)
					{
						string child = "bundle" + objects[i];
						if(i == 0)
							child = "cluster" + objects[0].Replace("cluster_", "").Replace(".txt", "");
						string parent = "bundle" + objects[i + 1];

						RoseTreeDT.Rows.Add(fieldID, child, parent);
					}
				}

				DataColumn[] keyColumns =
								new DataColumn[]{RoseTreeDT.Columns["Child"],
												RoseTreeDT.Columns["Parent"]};
				DataTableReviser.RemoveDuplicates(RoseTreeDT, keyColumns);
			}
		}

		private void LoadDocumentMatrix(string corpus1, string corpus2)
		{
			int corpusID1 = Corpora[corpus1];
			int corpusID2 = Corpora[corpus2];
			string docMatrxFolder = Path.Combine(folderName, string.Format("DocumentMatrix_{0}_{1}", corpus1, corpus2));

			var files = Directory.GetFiles(docMatrxFolder);
			foreach (var file in files)
			{
				//Console.WriteLine(">>LOG>> reading " + file + "...");
				LoadClusterDocumentMatrix(file, corpus1, corpus2);
			}
		}

		private void LoadClusterDocumentMatrix(string file, string corpus1, string corpus2)
		{
			int corpusID1 = Corpora[corpus1];
			int corpusID2 = Corpora[corpus2];
			using(StreamReader reader = new StreamReader(file))
			{
				var array = file.Split('\\');
				int clusterId = int.Parse(array[array.Length - 1].Replace("cluster", "").Replace(".txt", ""));

				while (true)
				{
					var line1 = reader.ReadLine();
					if (string.IsNullOrEmpty(line1))
						break;
					var line2 = reader.ReadLine();
					reader.ReadLine();

					var content1 = line1.Split(' ').Select(t=>t).Where(t=>t.Length > 0).ToArray();
					if (content1[0] != corpus1)
						throw new Exception("LoadClusterDocumentMatrix: corpus does not match.");
					int docId1 = int.Parse(content1[1]);

					var content2 = line2.Split(' ').Select(t=>t).Where(t=>t.Length > 0).ToArray();
					if (content2[0] != corpus2)
						throw new Exception("LoadClusterDocumentMatrix: corpus does not match.");
					for (int i = 1; i < content2.Length; i++)
					{
						var content = content2[i].Split(':');
						int docId2 = int.Parse(content[0]);
						double sim = double.Parse(content[1]);
						DocumentMatrixDT.Rows.Add(fieldID, clusterId, corpusID1, corpusID2, docId1, docId2, sim);
					}
				}
			}
		}

		private void LoadLocalLeadLagMeta(string corpus1, string corpus2)
		{
			int corpusID1 = Corpora[corpus1];
			int corpusID2 = Corpora[corpus2];
			string filename = Path.Combine(folderName, string.Format("LocalLeadLagSummary_{0}_{1}.txt", corpus1, corpus2));
			using (StreamReader reader = new StreamReader(filename))
			{
				while (true)
				{
					var topicline = reader.ReadLine();
					if (string.IsNullOrEmpty(topicline))
						break;
					var pair = topicline.Split(' ');
					int clusterId = int.Parse(pair[1]);
					while (true)
					{
						var line = reader.ReadLine();
						if (string.IsNullOrEmpty(line) || string.IsNullOrWhiteSpace(line))
							break;
						var contents = line.Split(' ').Select(t => t).Where(t => t.Length > 0).ToArray();
						int time = int.Parse(contents[0]);
						var timestamp = IntToDateTime(time);
						if(contents[1] != corpus1 || contents[3] != corpus2)
							throw new Exception("LoadLocalLeadLagMeta: corpus match error.");
						double value1 = double.Parse(contents[2]);
						double value2 = double.Parse(contents[4]);

						LocalLeadLagDT.Rows.Add(fieldID, clusterId, timestamp, corpusID1, corpusID2, value1, value2);
					}
				}
			}
		}

		private void LoadGlobalLeadLagMeta(string corpus1, string corpus2)
		{
			int corpusID1 = Corpora[corpus1];
			int corpusID2 = Corpora[corpus2];
			string filename = Path.Combine(folderName, string.Format("GLobalLeadLagSummary_{0}_{1}.txt", corpus1, corpus2));
			using (StreamReader reader = new StreamReader(filename))
			{
				while (true)
				{
					var level = reader.ReadLine();
					if (string.IsNullOrEmpty(level))
						break;
					while (true)
					{
						var line = reader.ReadLine();
						if (string.IsNullOrEmpty(line) || string.IsNullOrWhiteSpace(line))
							break;
						var contents = line.Split(' ').Select(t => t).Where(t => t.Length > 0).ToArray();
						string objectId = "";
						double leadlagValue = 0.0f;
						switch(level)
						{
							case "Topic": objectId = "topic" + int.Parse(contents[0]); leadlagValue = double.Parse(contents[4]); break;
							case "Bundle": objectId = "bundle" + int.Parse(contents[0]); leadlagValue = double.Parse(contents[4]); break;
							case "Global": objectId = "global"; leadlagValue = double.Parse(contents[3]); break;
						}
						string nonBaseCorpus = "";
						if (corpus1 == baseCorpus)
						{
							leadlagValue *= -1.0;
							nonBaseCorpus = corpus2;
						}
						else if (corpus2 == baseCorpus)
							nonBaseCorpus = corpus1;
						else
							throw new Exception("LoadGlobalLeadLagMeta: no base corpus.");
						GlobalLeadLagDT.Rows.Add(fieldID, objectId, Corpora[baseCorpus], 0.0);
						GlobalLeadLagDT.Rows.Add(fieldID, objectId, Corpora[nonBaseCorpus], leadlagValue);
					}
				}

				DataColumn[] keyColumns =
				new DataColumn[]{GlobalLeadLagDT.Columns["FieldID"],
								GlobalLeadLagDT.Columns["ObjectID"],
								GlobalLeadLagDT.Columns["CorpusID"]};
				DataTableReviser.RemoveDuplicates(GlobalLeadLagDT, keyColumns);
			}
		}
		#endregion

		private DateTime IntToDateTime(int value)
		{
			if (value < 10000)
				return new DateTime(value, 1, 1);
			else
				return new DateTime(value / 100, value % 100, 1);
		}
	}

	class AuthorNameParser
	{
		private static string UppercaseFirst(string s)
		{
			// Check for empty string.
			if (string.IsNullOrEmpty(s))
			{
				return string.Empty;
			}
			// Return char and concat substring.
			return char.ToUpper(s[0]) + s.Substring(1);
		}
		//new yang
		public static List<Tuple<string, string>> GetPaperAuthors(string field, string authors)
		{
			if (authors == "" || field == "")
			{
				return null;
			}
			else
			{
				List<Tuple<string, string>> toReturn = new List<Tuple<string, string>>();
				string[] authorArray = null;
				if (authors.Contains(",") && !authors.Contains(";"))
				{
					authorArray = authors.Split(new char[] { ',' });
				}
				else
				{
					authorArray = authors.Split(new char[] { ';' });
				}

				foreach (string au in authorArray)
				{
					if (au == "")
					{
						continue;
					}

					string[] names2 = au.Split(new char[] { ' ', '*', ','});
					List<string> names = new List<string>();
					for (int m = 0; m < names2.Length; m++)
					{
						if (names2[m] != "")
						{
							names.Add(names2[m]);
						}
					}
					if (names.Count == 0)
					{
						continue;
					}
					string first = "";
					string last = "";
					if (field == "infovis" && authors.Contains("*") == false)
					{
						GetInfovisPaperAuthor(au, ref first, ref last);
					}
					else if (authors.Contains("*") == true)
					{
						first = names[names.Count - 1];
						last = names[0];
					}
					else
					{
						first = names[0];
						last = names[names.Count - 1];
					}
					if (first == "" || last == "")
					{
						continue;
					}
					first = UppercaseFirst(first.Trim());
					last = UppercaseFirst(last.Trim());
					toReturn.Add(new Tuple<string, string>(first, last));
				}
				return toReturn;
			}
		}
		//new yang
		public static Tuple<string, string> GetPI(string pi)
		{
			if (pi == "")
			{
				return null;
			}
			else
			{
				string[] names = pi.Split(new char[] { ',' });
				if (names.Length != 2)
				{
					return null;
				}
				string last = UppercaseFirst(names[0].Trim());
				string first = names[1];
				if (first.Contains("."))
				{
					string[] fnames = first.Split(new char[] { '.' });
					if (fnames.Length > 0)
					{
						string s = fnames[names.Length - 1];
						if (s == "")
						{
							s = fnames[0];
						}

						first = s.Trim();
					}
				}
				else
				{
					first = first.Trim();
				}
				first = UppercaseFirst(first);
				Tuple<string, string> piname = new Tuple<string, string>(first, last);
				return piname;
			}
		}
		//new yang
		public static List<Tuple<string, string>> GetCoPI(string copi)
		{
			if (copi == "")
			{
				return null;
			}
			else
			{
				List<Tuple<string, string>> coPIList = new List<Tuple<string, string>>();
				string[] authorArray = copi.Split(new char[] { ';' });
				foreach (string au in authorArray)
				{
					if (au == "")
					{
						continue;
					}
					string[] names2 = au.Split(new char[] { ' ', '*' });
					List<string> names = new List<string>();
					foreach (string s in names2)
					{
						if (s == "" || s == " " || s.Contains("."))
						{
							continue;
						}
						names.Add(s);
					}
					if (names.Count == 2)
					{
						string first = names[0];
						first = UppercaseFirst(first.Trim());
						string last = names[1];
						last = UppercaseFirst(last.Trim());
						Tuple<string, string> copiName = new Tuple<string, string>(first, last);
						coPIList.Add(copiName);
					}
				}
				return coPIList;
			}
		}

		private static void GetInfovisPaperAuthor(string str, ref string first, ref string last)
		{
			if (str.Contains(","))
			{
				string[] names = str.Split(new char[] { ',' });
				if (names.Length > 1)
				{
					last = names[0].Trim();
					string f = names[1].Trim();
					first = f.Substring(0, 1);
				}
			}
			else
			{
				string[] names2 = str.Split(new char[] { ' ' });
				List<string> names = new List<string>();
				for (int m = 0; m < names2.Length; m++)
				{
					if (names2[m] != "")
					{
						names.Add(names2[m]);
					}
				}
				if (names.Count > 1)
				{
					first = names[0].Trim();
					last = names[names.Count - 1].Trim();
				}
			}

		}
	}
	class DataTableReviser
	{
		/// <summary>
		/// Removes duplicate rows from given DataTable
		/// </summary>
		/// <param name="tbl">Table to scan for duplicate rows</param>
		/// <param name="KeyColumns">An array of DataColumns
		///   containing the columns to match for duplicates</param>
		public static void RemoveDuplicates(DataTable tbl,
											 DataColumn[] keyColumns)
		{
			int rowNdx = 0;
			while (rowNdx < tbl.Rows.Count - 1)
			{
				DataRow[] dups = FindDups(tbl, rowNdx, keyColumns);
				if (dups.Length > 0)
				{
					foreach (DataRow dup in dups)
					{
						tbl.Rows.Remove(dup);
					}
				}
				else
				{
					rowNdx++;
				}
			}
		}

		private static DataRow[] FindDups(DataTable tbl,
										  int sourceNdx,
										  DataColumn[] keyColumns)
		{
			ArrayList retVal = new ArrayList();

			DataRow sourceRow = tbl.Rows[sourceNdx];
			for (int i = sourceNdx + 1; i < tbl.Rows.Count; i++)
			{
				DataRow targetRow = tbl.Rows[i];
				if (IsDup(sourceRow, targetRow, keyColumns))
				{
					retVal.Add(targetRow);
				}
			}
			return (DataRow[])retVal.ToArray(typeof(DataRow));
		}

		private static bool IsDup(DataRow sourceRow,
								  DataRow targetRow,
								  DataColumn[] keyColumns)
		{
			bool retVal = true;
			foreach (DataColumn column in keyColumns)
			{
				retVal = retVal && sourceRow[column].Equals(targetRow[column]);
				if (!retVal) break;
			}
			return retVal;
		}
	}
}
